'''
Created on 02/11/2012

@author: GabrielBacelar
'''

class HW5Q1(object):
    '''
    classdocs
    '''

    def __init__(self):
        '''
        Constructor
        '''
        
    def getHW5Q1(self):
        
        import QValueIteration
        self.qValueIteration = QValueIteration.QValueIteration()
        
        self.qValueIteration.rewardruns = 0
        self.qValueIteration.policyruns = 0
        self.qValueIteration.actionruns = 0
        
        self.qValueIteration.actions = ["Clockwise","Counterclockwise"]
        self.qValueIteration.nodes = ["A","B","C"]
        
        """
        Consider the following samples that the agent encountered. (Note that nan stands for not-a-number and indicates that this entry cannot be estimated from the samples.)
        s     a                     s'     r      |  s     a                     s'     r      |  s     a                     s'     r
        A     Clockwise             B     0.0    |  B     Clockwise             C     0.0    |  C     Clockwise             A     0.0
        A     Clockwise             B     0.0    |  B     Clockwise             C     0.0    |  C     Clockwise             B     -3.0
        A     Clockwise             B     0.0    |  B     Clockwise             C     0.0    |  C     Clockwise             B     -3.0
        A     Clockwise             C     -9.0   |  B     Clockwise             A     -4.0   |  C     Clockwise             A     0.0
        A     Clockwise             B     0.0    |  B     Clockwise             C     0.0    |  C     Clockwise             A     0.0
        A     Counterclockwise    C     -5.0   |  B     Counterclockwise    A     9.0    |  C     Counterclockwise    B     10.0
        A     Counterclockwise    C     -5.0   |  B     Counterclockwise    C     0.0    |  C     Counterclockwise    B     10.0
        A     Counterclockwise    C     -5.0   |  B     Counterclockwise    A     9.0    |  C     Counterclockwise    B     10.0
        A     Counterclockwise    B     0.0    |  B     Counterclockwise    C     0.0    |  C     Counterclockwise    A     0.0
        A     Counterclockwise    C     -5.0   |  B     Counterclockwise    C     0.0    |  C     Counterclockwise    B     10.0
        """
        self.qValueIteration.S = {}
        self.qValueIteration.S[("A","Clockwise","B")] = [0.0, 0.0, 0.0, 0.0]
        self.qValueIteration.S[("A","Clockwise","C")] = [-9.0]
        self.qValueIteration.S[("A","Counterclockwise","C")] = [-5.0, -5.0, -5.0, -5.0]
        self.qValueIteration.S[("A","Counterclockwise","B")] = [0.0]
        self.qValueIteration.S[("B","Clockwise","C")] = [0.0, 0.0, 0.0, 0.0]
        self.qValueIteration.S[("B","Clockwise","A")] = [-4.0]
        self.qValueIteration.S[("B","Counterclockwise","A")] = [9.0, 9.0]
        self.qValueIteration.S[("B","Counterclockwise","C")] = [0.0, 0.0, 0.0]
        self.qValueIteration.S[("C","Clockwise","A")] = [0.0, 0.0, 0.0]
        self.qValueIteration.S[("C","Clockwise","B")] = [-3.0, -3.0]
        self.qValueIteration.S[("C","Counterclockwise","A")] = [0.0]
        self.qValueIteration.S[("C","Counterclockwise","B")] = [10.0, 10.0, 10.0, 10.0]
        
    def getHW5Q1Part1(self):
        """
        Part 1: We start by estimating the transition function, T(s,a,s') and reward function R(s,a,s') for this MDP. Fill in the missing values in the following table for T(s,a,s') and R(s,a,s').
        Discount Factor, y = 0.5
        s     a                     s'     T(s,a,s')    R(s,a,s')
        A     Clockwise             B     M             N
        A     Clockwise             C     O             P
        A     Counterclockwise      B     0.200         0.000
        A     Counterclockwise      C     0.800         -5.000
        B     Clockwise             A     0.200         -4.000
        B     Clockwise             C     0.800         0.000
        B     Counterclockwise      A     0.400         9.000
        B     Counterclockwise      C     0.600         0.000
        C     Clockwise             A     0.600         0.000
        C     Clockwise             B     0.400         -3.000
        C     Counterclockwise      A     0.200         0.000
        C     Counterclockwise      B     0.800         10.000
        """
        self.qValueIteration.gama = 0.5
        
        self.qValueIteration.T = {}
        self.qValueIteration.T[("A","Clockwise","B")] = float('nan') #self.M # ->
        self.qValueIteration.T[("A","Clockwise","C")] = float('nan') #self.O # ->
        self.qValueIteration.T[("A","Counterclockwise","B")] = 0.200         
        self.qValueIteration.T[("A","Counterclockwise","C")] = 0.800         
        self.qValueIteration.T[("B","Clockwise","A")] = 0.200         
        self.qValueIteration.T[("B","Clockwise","C")] = 0.800         
        self.qValueIteration.T[("B","Counterclockwise","A")] = 0.400         
        self.qValueIteration.T[("B","Counterclockwise","C")] = 0.600         
        self.qValueIteration.T[("C","Clockwise","A")] = 0.600         
        self.qValueIteration.T[("C","Clockwise","B")] = 0.400         
        self.qValueIteration.T[("C","Counterclockwise","A")] = 0.200         
        self.qValueIteration.T[("C","Counterclockwise","B")] = 0.800         
        
        self.qValueIteration.R = {}
        self.qValueIteration.R[("A","Clockwise","B")] = float('nan') #self.N # ->
        self.qValueIteration.R[("A","Clockwise","C")] = float('nan') #self.P # ->
        self.qValueIteration.R[("A","Counterclockwise","B")] = 0.000
        self.qValueIteration.R[("A","Counterclockwise","C")] = -5.000
        self.qValueIteration.R[("B","Clockwise","A")] = -4.000
        self.qValueIteration.R[("B","Clockwise","C")] = 0.000
        self.qValueIteration.R[("B","Counterclockwise","A")] = 9.000
        self.qValueIteration.R[("B","Counterclockwise","C")] = 0.000
        self.qValueIteration.R[("C","Clockwise","A")] = 0.000
        self.qValueIteration.R[("C","Clockwise","B")] = -3.000
        self.qValueIteration.R[("C","Counterclockwise","A")] = 0.000
        self.qValueIteration.R[("C","Counterclockwise","B")] = 10.000
        
    def getHW5Q1Part2(self):
        """
        Part 2: Now we will run Q-iteration using the estimated T and R functions. The values of Qk(s,a), are given in the table below.
                            A       B       C
        Clockwise           0.44    2.22    -1.02
        Counterclockwise    -0.44   5.64    9.26
        """
        
        self.qValueIteration.Q = {}
        self.qValueIteration.Q["A","Clockwise"] = 0.44
        self.qValueIteration.Q["A","Counterclockwise"] = -0.44
        self.qValueIteration.Q["B","Clockwise"] = 2.22
        self.qValueIteration.Q["B","Counterclockwise"] = 5.64 
        self.qValueIteration.Q["C","Clockwise"] = -1.02
        self.qValueIteration.Q["C","Counterclockwise"] = 9.26 
        
    def getHW5Q1Part3(self):
        """
        Part 3: Suppose Q-iteration converges to the following Q* function, Q*(s,a).
                              A         B         C
        Clockwise             2.286     3.906     0.969
        Counterclockwise      1.219     7.416     11.195
        """
        
        self.qValueIteration.Qstar = {}
        self.qValueIteration.Qstar["A","Clockwise"] = 2.286
        self.qValueIteration.Qstar["A","Counterclockwise"] = 1.219
        self.qValueIteration.Qstar["B","Clockwise"] = 3.906
        self.qValueIteration.Qstar["B","Counterclockwise"] = 7.416
        self.qValueIteration.Qstar["C","Clockwise"] = 0.969
        self.qValueIteration.Qstar["C","Counterclockwise"] = 11.195
        
    def resolveHW5Q1(self):
        hw5q1.getHW5Q1()
        hw5q1.getHW5Q1Part1()
        
        print "Sample: ", hw5q1.qValueIteration.S
        print "Transactions: ", hw5q1.qValueIteration.T
        print "Rewards: ", hw5q1.qValueIteration.R
        print "Values: ", hw5q1.qValueIteration.V
        
        hw5q1.qValueIteration.getSampleAverages()
        
        print "Sample: ", hw5q1.qValueIteration.S
        print "Transactions: ", hw5q1.qValueIteration.T
        print "Rewards: ", hw5q1.qValueIteration.R
        print "Values: ", hw5q1.qValueIteration.V
        
        print "Part1: "
        print "T(A,Clockwise,B): ", self.qValueIteration.T[("A","Clockwise","B")]
        print "T(A,Clockwise,C): ", self.qValueIteration.T[("A","Clockwise","C")]
        print "R(A,Clockwise,B): ", self.qValueIteration.R[("A","Clockwise","B")]
        print "R(A,Clockwise,C): ", self.qValueIteration.R[("A","Clockwise","C")]
        
        
        #hw5q1.getHW5Q1Part2()
        
        print "T: ", self.qValueIteration.T
        print "R: ", self.qValueIteration.R
        print "Q: ", self.qValueIteration.Q
        
        hw5q1.qValueIteration.k = 100
        hw5q1.qValueIteration.getQValueItaration()

        """        
        T(A,Clockwise,B):  0.8
        T(A,Clockwise,C):  0.2
        R(A,Clockwise,B):  0.0
        R(A,Clockwise,C):  -9.0
        
        QValue for k  :  98 {
        ('A', 'Counterclockwise'): -0.44                -> -0.44
        ('A', 'Clockwise'): 0.44000000000000017            -> 0.44
        ('B', 'Clockwise'): 2.22                        -> 2.22
        ('B', 'Counterclockwise'): 5.640000000000001    -> 5.64
        ('C', 'Clockwise'): -1.02                         -> -1.02
        ('C', 'Counterclockwise'): 9.260000000000002    -> 9.26
        }
        
        QValue for k+1:  100 {
        ('A', 'Clockwise'): 1.3819999999999997             -> 1.382
        ('A', 'Counterclockwise'): 0.26799999999999985     -> 0.268
        ('B', 'Clockwise'): 2.9480000000000004            -> 2.948
        ('B', 'Counterclockwise'): 6.466000000000001     -> 6.466
        ('C', 'Clockwise'): 0.05999999999999994         -> 0.06
        ('C', 'Counterclockwise'): 10.3                 -> 10.3
        }
        
        QValue for k+1:  1 {
        ('A', 'Clockwise'): 2.285714285714286             -> 2.286
        ('A', 'Counterclockwise'): 1.2194805194805196    -> 1.219 
        ('B', 'Clockwise'): 3.906493506493507            -> 3.906
        ('B', 'Counterclockwise'): 7.415584415584416     -> 7.416
        ('C', 'Clockwise'): 0.968831168831169             -> 0.969
        ('C', 'Counterclockwise'): 11.194805194805195     -> 11.195

        """
        
hw5q1 = HW5Q1()
hw5q1.resolveHW5Q1()

#print "OK", hw5q1.qValueIteration.S
#hw5q1.resolveHW5Q1()
        